#!/usr/bin/env python
#  Copyright (c) 2010
#  The Regents of the University of Michigan
#  All Rights Reserved

#  Permission is granted to use, copy, create derivative works, and
#  redistribute this software and such derivative works for any purpose,
#  so long as the name of the University of Michigan is not used in
#  any advertising or publicity pertaining to the use or distribution
#  of this software without specific, written prior authorization. If
#  the above copyright notice or any other identification of the
#  University of Michigan is included in any copy of any portion of
#  this software, then the disclaimer below must also be included.

#  This software is provided as is, without representation or warranty
#  of any kind either express or implied, including without limitation
#  the implied warranties of merchantability, fitness for a particular
#  purpose, or noninfringement.  The Regents of the University of
#  Michigan shall not be liable for any damages, including special,
#  indirect, incidental, or consequential damages, with respect to any
#  claim arising out of or in connection with the use of the software,
#  even if it has been or is hereafter advised of the possibility of
#  such damages.

import bencode
import log_parser

import collections
import cPickle
import socket  # For inet_aton.
import sys
import time

def is_peer_record(s):
  '''Return True iff s is a peer record.
  Peer record format: [ip_override:]tcp_port[;C][;udp_port]
  '''

  if len(s) > 29:
    return False

  colon_idx = s.find(':')
  if colon_idx != -1:
    ip_part = s[:colon_idx]
    try:
      socket.inet_aton(ip_part)
    except socket.error:
      return False
    except TypeError:
      return False
    s = s[colon_idx+1:]
  
  first_semicolon_idx = s.find(';')
  if first_semicolon_idx == -1:
    first_semicolon_idx = len(s)

  # Check the TCP port.
  try:
    if not 0 <= int(s[:first_semicolon_idx]) <= 65535:
      return False
  except ValueError:
    return False


  s = s[first_semicolon_idx+1:]
  if not s:
    return False

  if s == 'C':
    return True

  if s.startswith('C;'):
    s = s[2:]
    # Check the UDP port.
  try:
    if not 0 <= int(s) <= 65535:
      return False
  except ValueError:
    return False

  return True
    

def main():
  pickle_file_name = '/home/swolchok/10klongrununiquekvpairs/%s.uniquekvpairs.pickle' % sys.argv[1]
  print 'Pickle file: %s' % pickle_file_name
  unique_key_value_pairs = collections.defaultdict(set)
  for log in sys.argv[2:]:
    log_list = log_parser.get_stores(log)
    if not log_list:
      continue
    for (key, store_time, version, value) in log_list:
      if ((len(value) <= 5 and value.isdigit()) or
          bencode.simple_is_bencoded(value) or
          is_peer_record(value)):
        continue
      unique_key_value_pairs[key].add(value)
#          print 'Key %s FOUND in %s, stored at %s' % (key.encode('hex'), log, store_time)
#        else:
#          print 'Key %s found with wrong value %s instead of %s' %(key.encode('hex'), TARGET_KEYS[key].encode('hex'), value.encode('hex'))

  out = dict(unique_key_value_pairs.iteritems())
  del unique_key_value_pairs
  cPickle.dump(out, open(pickle_file_name, 'wb'))
if __name__ == '__main__':
  main()
